#!/usr/bin/env python
# -*- coding:utf-8 -*-

import requests
import re
import os
from optparse import OptionParser


def get_jandown(s, re_jd, html_data_url, proxies):
	r = s.get(html_data_url, proxies = proxies, verify=False)
	content = r.content
	re_jd_result = re_jd.search(content)
	return re_jd_result.group()


def get_html(s, re_htm, start_url, proxies):
	r = s.get(start_url, proxies=proxies, verify=False)
	content = r.content
	re_htm_result = re_htm.findall(content)
	return list(set(re_htm_result))


# check if it is needed to use proxy
parser = OptionParser(usage="usage:%prog [optinos] filepath")
parser.add_option("-p", "--proxy",
                  action="store_true",
                  dest="proxy",
                  default=True,
                  help="flags about use proxy default True")
parser.add_option("-n", '--page',
                  action="store",
                  dest="page",
                  default="1",
                  help="page num")
(options, args) = parser.parse_args()


if not options.proxy:
	proxies = dict()
else:
	proxies = {
		'http': 'http://127.0.0.1:1080',
		'https': 'http://127.0.0.1:1080',
	}

# page url
if not options.page:
	start_url = 'http://ac88.info/bt/thread.php?fid=16'
else:
	start_url = 'http://ac88.info/bt/thread.php?fid=16&page='+options.page

# item detail page url prefix
item_prefix = 'http://ac88.info/bt/'

re_str_jd = '(?<=>)http://www.jandown.com[^<>"]+'
re_jd = re.compile(re_str_jd)

re_str_htm = 'htm_[^"]+'
re_htm = re.compile(re_str_htm)


# setting charset
s = requests.Session()
s.encoding = 'gbk'

htm_url_list = get_html(s, re_htm, start_url, proxies)
jd_url_list = []
index = 0

for htm_url in htm_url_list:
	index += 1
	full_htm_url = ''
	try:
		full_htm_url = item_prefix + htm_url
		jd_url = get_jandown(s, re_jd, full_htm_url, proxies)
		jd_url_list.insert(-1, jd_url)
	except AttributeError:
		print full_htm_url
	print 'finished %d ...' % index


full_path = os.path.join(os.getcwd(), "acinfo.list")
with open(full_path, 'w') as fid:
	fid.write('\n'.join(jd_url_list))

